# encoding: UTF-8
# user：admin at 2017/4/24

import re
from openpyxl import Workbook
import requests
from HTMLParser import HTMLParser

def crawl_douban_book_list(page_url):
    #url="https://book.douban.com/top250?start="+str(iPage)
    res=requests.get(page_url)

    #获取整个数据的正则
    pattern=re.compile("<div class=\"indent\">.*?</div>",re.S)
    #<div class="indent">
    html_parser=HTMLParser()
    #将html中的换行符转换成\n的格式
    htmlPage=html_parser.unescape(res.text).replace("<br/>","\n")
    #print htmlPage
    #body=pattern.findall(htmlPage)
    #print body
    htmlPage=htmlPage
    #每本书的正则表达式
    m_pattern=re.compile(u"<table width=\"100%\">(.*?)</table>",re.S)
    m=m_pattern.findall(htmlPage)

    #书名的正则 中文 英文
    title_pattern=re.compile(u"<div class=\"pl2\">.*?<a href=.*? onclick=.*? title=.*?>(.*?)</a>",re.S)

    #作者，出版社信息的正则
    author_pattern=re.compile("<p class=\"pl\">(.*?)</p>",re.S)

    #评分的正则
    rate_pattern=re.compile("<div class=\"star clearfix\">.*?<span class=\"rating_nums\">(.*?)</span>",re.S)

    #评论数量的正则
    pl_pattern=re.compile("<div class=\"star clearfix\">.*?<span class=\"pl\">\((.*?)\)</span>",re.S)

    #短评的正则
    quote_pattern=re.compile("<span class=\"inq\">(.*?)</span>",re.S)

    #处理副标题的正则
    span_pattern=re.compile("<span style=.*?>(.*?)</span>",re.S)
    titleMain_pattern=re.compile("(.*?)<span .*?>.*?</span>",re.S)

    title_list=[]
    author_list=[]
    rate_list=[]
    pl_list=[]
    quote_list=[]

    year_list=[]
    price_list=[]
    pulish_list=[]
    tranName_list=[]

    for book in m:
        #书名
        title=title_pattern.findall(book)
        #作者信息
        author=author_pattern.findall(book)
        #评分
        rate=rate_pattern.findall(book)
        #评论数量
        pl=pl_pattern.findall(book)
        #短评
        quote=quote_pattern.findall(book)

        titleName = title[0].replace("\n", "").strip()
        #print bookname.find('span')
        span_text=""
        #处理有副标题的图书名字
        if titleName.find('<span')>0:     #查找字符串中是否存在span的
            span=span_pattern.findall(titleName)
            title_main=titleMain_pattern.findall(titleName)
            #print title_main[0],span[0]
            span_text=span[0].strip()       #副标题
            bookname=title_main[0].strip()+span_text

        else:
            bookname = title[0].replace("\n", "").strip()

        title_list.append(bookname)

        zuozhe=author[0].split("/")
        print zuozhe[-1],zuozhe[-2],zuozhe[-3],len(zuozhe)

        price=zuozhe[-1].strip()    #价格
        year=zuozhe[-2].strip()     #年份
        pulish=zuozhe[-3].strip()   #出版社
        #authorName=""
        authorName = zuozhe[0].strip()  #作者
        tranName=""     #翻译人
        if len(zuozhe)==5:      #表示国外的图书
            print u"国外的书"
            #authorName=zuozhe[1]
            tranName=zuozhe[1].strip()

        else:   #表示国内的图书
            print u"国内的书"

        year_list.append(year)
        price_list.append(price)
        pulish_list.append(pulish)
        tranName_list.append(tranName)

        author_list.append(authorName)

        rate_list.append(rate[0])

        pl_num=pl[0].replace("\n","").strip()
        pl_list.append(pl_num)

        if quote:
            quote_list.append(quote[0])
        else:
            quote_list.append("无")

        # print title[0].replace("\n","").strip()
        # print author[0],rate[0]
        # print pl[0].replace("\n","").strip(),quote[0]

    return title_list,author_list,rate_list,pl_list,quote_list,year_list,price_list,pulish_list,tranName_list

def main():
    fileName=u"豆瓣图书top250.xlsx"
    wb=Workbook()
    ws=wb.active
    ws.title=u"图书top250"
    url = "https://book.douban.com/top250?"

    iCnt=1

    ws.cell("A1").value="书名"
    ws.cell("B1").value = "作者信息"
    ws.cell("C1").value = "翻译者"
    ws.cell("D1").value = "出版社"
    ws.cell("E1").value = "出版日期"
    ws.cell("F1").value = "评分"
    ws.cell("G1").value = "评论数量"
    ws.cell("H1").value = "短评"

    title_list=[]
    author_list=[]
    rate_list=[]
    pl_list=[]
    quote_list=[]

    year_list=[]
    price_list=[]
    pulish_list=[]
    tranName_list=[]

    for i in range(0,10):
        page_value="start="+str(i*25)   #i*25
        #?start=225&filter=
        page_url=url+page_value

        title_list, author_list, rate_list, pl_list, quote_list,year_list,price_list,pulish_list,tranName_list=crawl_douban_book_list(page_url)

        for (title,author,rate,pl,quote,year,price,pulish,tranName) in zip(title_list, author_list, rate_list, pl_list, quote_list,year_list,price_list,pulish_list,tranName_list):
            iCnt += 1
            col_A = "A"+str(iCnt)
            col_B = "B"+str(iCnt)
            col_C = "C"+str(iCnt)
            col_D = "D"+str(iCnt)
            col_E = "E"+str(iCnt)
            col_F = "F"+str(iCnt)
            col_G = "G"+str(iCnt)
            col_H = "H"+str(iCnt)

            ws.cell(col_A).value=title
            ws.cell(col_B).value = author

            ws.cell(col_C).value = tranName
            ws.cell(col_D).value = pulish
            ws.cell(col_E).value = year

            ws.cell(col_F).value = rate
            ws.cell(col_G).value = pl
            ws.cell(col_H).value = quote

    wb.save(filename=fileName)

if __name__ == '__main__':
    main()